{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = np.load('data/x_train.npy')\n",
    "y_train = np.load('data/y_train.npy')\n",
    "x_test = np.load('data/x_test.npy')\n",
    "y_test = np.load('data/y_test.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5344, 224, 224, 3)\n",
      "(654, 224, 224, 3)\n"
     ]
    }
   ],
   "source": [
    "print(x_train.shape)\n",
    "print(x_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5344,)\n",
      "(654,)\n"
     ]
    }
   ],
   "source": [
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#x_train /= 255????"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "\n",
    "import numpy as np\n",
    "from random import shuffle\n",
    "import time\n",
    "import csv\n",
    "from PIL import Image\n",
    "import os\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "from keras.callbacks import EarlyStopping, LearningRateScheduler\n",
    "from keras import initializers\n",
    "from keras.optimizers import SGD\n",
    "from keras.preprocessing import sequence\n",
    "from keras.utils import np_utils\n",
    "from keras.models import Sequential,load_model,Model\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import *\n",
    "from keras.callbacks import CSVLogger\n",
    "from keras import callbacks\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.model_selection import train_test_split\n",
    "import sklearn.metrics as sklm\n",
    "import lossprettifier\n",
    "from VGG import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for reproducibility\n",
    "np.random.seed(3768)\n",
    "\n",
    "# use this environment flag to change which GPU to use \n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"  # specify which GPU(s) to be used\n",
    "\n",
    "#Get TensorFlow session\n",
    "def get_session(): \n",
    "  config = tf.ConfigProto() \n",
    "  config.gpu_options.allow_growth = True \n",
    "  return tf.Session(config=config) \n",
    "  \n",
    "# One hot encoding of labels \n",
    "def dense_to_one_hot(labels_dense,num_clases=4):\n",
    "  return np.eye(num_clases)[labels_dense]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preparing training and test sets\n",
    "x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.10, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = dense_to_one_hot(y_train,num_clases=4)\n",
    "y_valid= dense_to_one_hot(y_valid,num_clases=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/reza/.local/lib/python3.6/site-packages/keras_preprocessing/image/image_data_generator.py:348: UserWarning: This ImageDataGenerator specifies `featurewise_std_normalization`, which overrides setting of `featurewise_center`.\n",
      "  warnings.warn('This ImageDataGenerator specifies '\n"
     ]
    }
   ],
   "source": [
    "#Image data generation for the training \n",
    "datagen = ImageDataGenerator(\n",
    "               featurewise_center = False, \n",
    "               samplewise_center = False,  # set each sample mean to 0\n",
    "               featurewise_std_normalization = True,  \n",
    "               samplewise_std_normalization = False)  \n",
    "\n",
    "datagen.fit(x_train) \n",
    "for i in range(len(x_test)):\n",
    "      x_test[i] = datagen.standardize(x_test[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "32/32 [==============================] - 20s 627ms/step - loss: 1.1260 - accuracy: 0.4854 - val_loss: 0.8739 - val_accuracy: 0.5586\n",
      "Epoch 2/50\n",
      "32/32 [==============================] - 19s 602ms/step - loss: 0.9745 - accuracy: 0.5459 - val_loss: 0.7980 - val_accuracy: 0.6620\n",
      "Epoch 3/50\n",
      "32/32 [==============================] - 19s 591ms/step - loss: 0.8714 - accuracy: 0.6054 - val_loss: 0.6983 - val_accuracy: 0.6700\n",
      "Epoch 4/50\n",
      "32/32 [==============================] - 20s 609ms/step - loss: 0.7564 - accuracy: 0.6807 - val_loss: 0.4598 - val_accuracy: 0.7654\n",
      "Epoch 5/50\n",
      "32/32 [==============================] - 19s 604ms/step - loss: 0.7233 - accuracy: 0.7168 - val_loss: 0.9107 - val_accuracy: 0.7256\n",
      "Epoch 6/50\n",
      "32/32 [==============================] - 19s 592ms/step - loss: 0.6709 - accuracy: 0.7083 - val_loss: 0.6225 - val_accuracy: 0.7316\n",
      "Epoch 7/50\n",
      "32/32 [==============================] - 19s 593ms/step - loss: 0.6825 - accuracy: 0.7139 - val_loss: 0.4330 - val_accuracy: 0.7654\n",
      "Epoch 8/50\n",
      "32/32 [==============================] - 19s 605ms/step - loss: 0.7340 - accuracy: 0.7041 - val_loss: 0.6042 - val_accuracy: 0.7097\n",
      "Epoch 9/50\n",
      "32/32 [==============================] - 19s 604ms/step - loss: 0.7123 - accuracy: 0.6992 - val_loss: 0.3131 - val_accuracy: 0.7416\n",
      "Epoch 10/50\n",
      "32/32 [==============================] - 20s 621ms/step - loss: 0.6631 - accuracy: 0.7266 - val_loss: 0.7513 - val_accuracy: 0.7555\n",
      "Epoch 11/50\n",
      "32/32 [==============================] - 20s 612ms/step - loss: 0.6787 - accuracy: 0.7041 - val_loss: 0.5781 - val_accuracy: 0.7575\n",
      "Epoch 12/50\n",
      "32/32 [==============================] - 20s 612ms/step - loss: 0.6753 - accuracy: 0.7285 - val_loss: 0.6944 - val_accuracy: 0.7654\n",
      "Epoch 13/50\n",
      "32/32 [==============================] - 19s 608ms/step - loss: 0.6375 - accuracy: 0.7413 - val_loss: 0.4953 - val_accuracy: 0.7674\n",
      "Epoch 14/50\n",
      "32/32 [==============================] - 19s 601ms/step - loss: 0.6257 - accuracy: 0.7314 - val_loss: 0.4217 - val_accuracy: 0.7614\n",
      "Epoch 15/50\n",
      "32/32 [==============================] - 19s 592ms/step - loss: 0.5966 - accuracy: 0.7532 - val_loss: 0.5771 - val_accuracy: 0.7694\n",
      "Epoch 16/50\n",
      "32/32 [==============================] - 19s 606ms/step - loss: 0.5862 - accuracy: 0.7588 - val_loss: 0.5979 - val_accuracy: 0.7773\n",
      "Epoch 17/50\n",
      "32/32 [==============================] - 19s 608ms/step - loss: 0.6080 - accuracy: 0.7471 - val_loss: 0.5961 - val_accuracy: 0.7594\n",
      "Epoch 18/50\n",
      "32/32 [==============================] - 19s 600ms/step - loss: 0.6193 - accuracy: 0.7266 - val_loss: 0.6281 - val_accuracy: 0.7617\n",
      "Epoch 19/50\n",
      "32/32 [==============================] - 20s 615ms/step - loss: 0.6069 - accuracy: 0.7363 - val_loss: 0.4915 - val_accuracy: 0.7256\n",
      "Epoch 20/50\n",
      "32/32 [==============================] - 20s 615ms/step - loss: 0.5706 - accuracy: 0.7588 - val_loss: 0.4789 - val_accuracy: 0.7873\n",
      "Epoch 21/50\n",
      "32/32 [==============================] - 19s 587ms/step - loss: 0.5736 - accuracy: 0.7646 - val_loss: 0.5547 - val_accuracy: 0.7694\n",
      "Epoch 22/50\n",
      "32/32 [==============================] - 19s 602ms/step - loss: 0.6193 - accuracy: 0.7373 - val_loss: 0.6269 - val_accuracy: 0.7952\n",
      "Epoch 23/50\n",
      "32/32 [==============================] - 19s 601ms/step - loss: 0.5919 - accuracy: 0.7588 - val_loss: 0.7170 - val_accuracy: 0.7893\n",
      "Epoch 24/50\n",
      "32/32 [==============================] - 19s 592ms/step - loss: 0.5902 - accuracy: 0.7532 - val_loss: 0.4764 - val_accuracy: 0.7674\n",
      "Epoch 25/50\n",
      "32/32 [==============================] - 20s 620ms/step - loss: 0.5882 - accuracy: 0.7559 - val_loss: 0.6773 - val_accuracy: 0.8131\n",
      "Epoch 26/50\n",
      "32/32 [==============================] - 19s 597ms/step - loss: 0.5588 - accuracy: 0.7695 - val_loss: 0.6876 - val_accuracy: 0.7873\n",
      "Epoch 27/50\n",
      "32/32 [==============================] - 19s 606ms/step - loss: 0.5676 - accuracy: 0.7656 - val_loss: 0.3456 - val_accuracy: 0.7694\n",
      "Epoch 28/50\n",
      "32/32 [==============================] - 20s 617ms/step - loss: 0.6506 - accuracy: 0.7213 - val_loss: 0.6469 - val_accuracy: 0.7515\n",
      "Epoch 29/50\n",
      "32/32 [==============================] - 19s 603ms/step - loss: 0.6499 - accuracy: 0.7113 - val_loss: 0.3253 - val_accuracy: 0.7753\n",
      "Epoch 30/50\n",
      "32/32 [==============================] - 19s 597ms/step - loss: 0.5674 - accuracy: 0.7656 - val_loss: 0.6697 - val_accuracy: 0.7674\n",
      "Epoch 31/50\n",
      "32/32 [==============================] - 20s 612ms/step - loss: 0.6147 - accuracy: 0.7559 - val_loss: 0.6267 - val_accuracy: 0.7654\n",
      "Epoch 32/50\n",
      "32/32 [==============================] - 20s 610ms/step - loss: 0.5915 - accuracy: 0.7295 - val_loss: 0.6370 - val_accuracy: 0.7654\n",
      "Epoch 33/50\n",
      "32/32 [==============================] - 19s 588ms/step - loss: 0.5373 - accuracy: 0.7666 - val_loss: 0.5714 - val_accuracy: 0.7714\n",
      "Epoch 34/50\n",
      "32/32 [==============================] - 20s 610ms/step - loss: 0.5659 - accuracy: 0.7637 - val_loss: 0.6328 - val_accuracy: 0.7674\n",
      "Epoch 35/50\n",
      "32/32 [==============================] - 20s 613ms/step - loss: 0.5549 - accuracy: 0.7562 - val_loss: 1.1637 - val_accuracy: 0.7734\n",
      "Epoch 36/50\n",
      "32/32 [==============================] - 20s 620ms/step - loss: 0.5715 - accuracy: 0.7412 - val_loss: 0.3932 - val_accuracy: 0.7932\n",
      "Epoch 37/50\n",
      "32/32 [==============================] - 20s 614ms/step - loss: 0.5887 - accuracy: 0.7559 - val_loss: 0.5093 - val_accuracy: 0.7674\n",
      "Epoch 38/50\n",
      "32/32 [==============================] - 19s 602ms/step - loss: 0.5466 - accuracy: 0.7539 - val_loss: 0.5148 - val_accuracy: 0.7495\n",
      "Epoch 39/50\n",
      "32/32 [==============================] - 20s 614ms/step - loss: 0.4989 - accuracy: 0.7959 - val_loss: 0.4625 - val_accuracy: 0.7873\n",
      "Epoch 40/50\n",
      "32/32 [==============================] - 20s 618ms/step - loss: 0.5473 - accuracy: 0.7773 - val_loss: 0.7144 - val_accuracy: 0.7853\n",
      "Epoch 41/50\n",
      "32/32 [==============================] - 19s 584ms/step - loss: 0.5425 - accuracy: 0.7637 - val_loss: 0.7769 - val_accuracy: 0.7495\n",
      "Epoch 42/50\n",
      "32/32 [==============================] - 19s 609ms/step - loss: 0.5811 - accuracy: 0.7490 - val_loss: 0.6224 - val_accuracy: 0.7833\n",
      "Epoch 43/50\n",
      "32/32 [==============================] - 19s 587ms/step - loss: 0.5636 - accuracy: 0.7453 - val_loss: 0.7475 - val_accuracy: 0.7416\n",
      "Epoch 44/50\n",
      "32/32 [==============================] - 20s 619ms/step - loss: 0.5788 - accuracy: 0.7529 - val_loss: 0.5530 - val_accuracy: 0.7893\n",
      "Epoch 45/50\n",
      "32/32 [==============================] - 19s 607ms/step - loss: 0.5080 - accuracy: 0.7812 - val_loss: 0.6491 - val_accuracy: 0.7813\n",
      "Epoch 46/50\n",
      "32/32 [==============================] - 20s 613ms/step - loss: 0.5433 - accuracy: 0.7764 - val_loss: 0.5392 - val_accuracy: 0.7893\n",
      "Epoch 47/50\n",
      "32/32 [==============================] - 20s 612ms/step - loss: 0.4825 - accuracy: 0.7783 - val_loss: 0.6117 - val_accuracy: 0.7913\n",
      "Epoch 48/50\n",
      "32/32 [==============================] - 20s 618ms/step - loss: 0.4966 - accuracy: 0.7705 - val_loss: 0.4517 - val_accuracy: 0.7972\n",
      "Epoch 49/50\n",
      "32/32 [==============================] - 19s 602ms/step - loss: 0.5399 - accuracy: 0.7764 - val_loss: 0.5197 - val_accuracy: 0.8012\n",
      "Epoch 50/50\n",
      "32/32 [==============================] - 19s 593ms/step - loss: 0.5231 - accuracy: 0.7832 - val_loss: 0.5512 - val_accuracy: 0.7773\n"
     ]
    }
   ],
   "source": [
    "#Defining hyperparameters\n",
    "batch_Size = 32\n",
    "steps_Per_Epoch = 32\n",
    "numEpochs = 50\n",
    "\n",
    "#Instantating VGG19 model\n",
    "model = VGG19((224,224,3),4) #VGG19_dense for revised VGG19, VGG19 for VGG19. Please pay attention to VGG16(), chnage the input shape and class number in VGG.py.\n",
    "\n",
    "#Creating an optimizers\n",
    "adaDelta = keras.optimizers.Adadelta(lr=1.0, rho=0.95)\n",
    "sgd = SGD(lr=0.01, decay=1e-6, momentum=0.95, nesterov=True)\n",
    "model.compile(optimizer = sgd , loss = 'categorical_crossentropy', metrics = ['accuracy'])\n",
    "\n",
    "#Creating early stopping \n",
    "earlystop = EarlyStopping(monitor = 'val_accuracy', min_delta = 0, patience = 50, verbose = 1, mode = 'auto', restore_best_weights = True)       \n",
    "\n",
    "train_generator = datagen.flow(x_train, y_train, batch_size = batch_Size)\n",
    "validation_generator = datagen.flow(x_valid, y_valid, batch_size = batch_Size)\n",
    "\n",
    "# Model training\n",
    "history = model.fit_generator(\n",
    "    train_generator,\n",
    "    steps_per_epoch = steps_Per_Epoch,\n",
    "    validation_data = validation_generator, \n",
    "    validation_steps = 16,\n",
    "    epochs = numEpochs,\n",
    "    shuffle = True, \n",
    "    verbose = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelPath = \"VGG19_COVID19.h5\"\n",
    "resultPath = 'VGG19_COVID19.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch     0 | LossA: 1.13(+0.00%) \u001b[0m\t| LossAB: 0.87(+0.00%) \u001b[0m\t\n",
      "Epoch     1 | LossA: \u001b[32m0.97(-13.46%) ▼\u001b[0m\t| LossAB: \u001b[32m0.80(-8.68%) ▼\u001b[0m\t\n",
      "Epoch     2 | LossA: \u001b[32m0.88(-9.63%) ▼\u001b[0m\t| LossAB: \u001b[32m0.70(-12.50%) ▼\u001b[0m\t\n",
      "Epoch     3 | LossA: \u001b[32m0.76(-14.12%) ▼\u001b[0m\t| LossAB: \u001b[32m0.46(-34.15%) ▼\u001b[0m\t\n",
      "Epoch     4 | LossA: \u001b[32m0.72(-4.37%) ▼\u001b[0m\t| LossAB: \u001b[91m0.91(+98.05%) ▲\u001b[0m\t\n",
      "Epoch     5 | LossA: \u001b[32m0.68(-5.74%) ▼\u001b[0m\t| LossAB: \u001b[32m0.62(-31.65%) ▼\u001b[0m\t\n",
      "Epoch     6 | LossA: \u001b[91m0.68(+0.10%) ▲\u001b[0m\t| LossAB: \u001b[32m0.43(-30.45%) ▼\u001b[0m\t\n",
      "Epoch     7 | LossA: \u001b[91m0.73(+7.55%) ▲\u001b[0m\t| LossAB: \u001b[91m0.60(+39.55%) ▲\u001b[0m\t\n",
      "Epoch     8 | LossA: \u001b[32m0.71(-2.96%) ▼\u001b[0m\t| LossAB: \u001b[32m0.31(-48.18%) ▼\u001b[0m\t\n",
      "Epoch     9 | LossA: \u001b[32m0.66(-6.91%) ▼\u001b[0m\t| LossAB: \u001b[91m0.75(+139.94%) ▲\u001b[0m\t\n",
      "Epoch    10 | LossA: \u001b[91m0.68(+2.35%) ▲\u001b[0m\t| LossAB: \u001b[32m0.58(-23.05%) ▼\u001b[0m\t\n",
      "Epoch    11 | LossA: \u001b[32m0.68(-0.49%) ▼\u001b[0m\t| LossAB: \u001b[91m0.69(+20.11%) ▲\u001b[0m\t\n",
      "Epoch    12 | LossA: \u001b[32m0.64(-5.25%) ▼\u001b[0m\t| LossAB: \u001b[32m0.50(-28.67%) ▼\u001b[0m\t\n",
      "Epoch    13 | LossA: \u001b[32m0.63(-2.21%) ▼\u001b[0m\t| LossAB: \u001b[32m0.42(-14.85%) ▼\u001b[0m\t\n",
      "Epoch    14 | LossA: \u001b[32m0.60(-4.87%) ▼\u001b[0m\t| LossAB: \u001b[91m0.58(+36.84%) ▲\u001b[0m\t\n",
      "Epoch    15 | LossA: \u001b[32m0.59(-1.51%) ▼\u001b[0m\t| LossAB: \u001b[91m0.60(+3.61%) ▲\u001b[0m\t\n",
      "Epoch    16 | LossA: \u001b[91m0.61(+3.71%) ▲\u001b[0m\t| LossAB: \u001b[32m0.60(-0.29%) ▼\u001b[0m\t\n",
      "Epoch    17 | LossA: \u001b[91m0.62(+1.86%) ▲\u001b[0m\t| LossAB: \u001b[91m0.63(+5.36%) ▲\u001b[0m\t\n",
      "Epoch    18 | LossA: \u001b[32m0.61(-2.00%) ▼\u001b[0m\t| LossAB: \u001b[32m0.49(-21.74%) ▼\u001b[0m\t\n",
      "Epoch    19 | LossA: \u001b[32m0.57(-5.98%) ▼\u001b[0m\t| LossAB: \u001b[32m0.48(-2.56%) ▼\u001b[0m\t\n",
      "Epoch    20 | LossA: \u001b[91m0.57(+0.54%) ▲\u001b[0m\t| LossAB: \u001b[91m0.55(+15.82%) ▲\u001b[0m\t\n",
      "Epoch    21 | LossA: \u001b[91m0.62(+7.96%) ▲\u001b[0m\t| LossAB: \u001b[91m0.63(+13.00%) ▲\u001b[0m\t\n",
      "Epoch    22 | LossA: \u001b[32m0.59(-4.43%) ▼\u001b[0m\t| LossAB: \u001b[91m0.72(+14.39%) ▲\u001b[0m\t\n",
      "Epoch    23 | LossA: \u001b[32m0.59(-0.60%) ▼\u001b[0m\t| LossAB: \u001b[32m0.48(-33.56%) ▼\u001b[0m\t\n",
      "Epoch    24 | LossA: \u001b[32m0.59(-0.02%) ▼\u001b[0m\t| LossAB: \u001b[91m0.68(+42.17%) ▲\u001b[0m\t\n",
      "Epoch    25 | LossA: \u001b[32m0.56(-5.00%) ▼\u001b[0m\t| LossAB: \u001b[91m0.69(+1.53%) ▲\u001b[0m\t\n",
      "Epoch    26 | LossA: \u001b[91m0.57(+1.58%) ▲\u001b[0m\t| LossAB: \u001b[32m0.35(-49.74%) ▼\u001b[0m\t\n",
      "Epoch    27 | LossA: \u001b[91m0.65(+14.08%) ▲\u001b[0m\t| LossAB: \u001b[91m0.65(+87.21%) ▲\u001b[0m\t\n",
      "Epoch    28 | LossA: \u001b[91m0.65(+0.16%) ▲\u001b[0m\t| LossAB: \u001b[32m0.33(-49.71%) ▼\u001b[0m\t\n",
      "Epoch    29 | LossA: \u001b[32m0.57(-12.52%) ▼\u001b[0m\t| LossAB: \u001b[91m0.67(+105.85%) ▲\u001b[0m\t\n",
      "Epoch    30 | LossA: \u001b[91m0.61(+8.35%) ▲\u001b[0m\t| LossAB: \u001b[32m0.63(-6.41%) ▼\u001b[0m\t\n",
      "Epoch    31 | LossA: \u001b[32m0.59(-3.78%) ▼\u001b[0m\t| LossAB: \u001b[91m0.64(+1.64%) ▲\u001b[0m\t\n",
      "Epoch    32 | LossA: \u001b[32m0.54(-9.16%) ▼\u001b[0m\t| LossAB: \u001b[32m0.57(-10.30%) ▼\u001b[0m\t\n",
      "Epoch    33 | LossA: \u001b[91m0.57(+5.32%) ▲\u001b[0m\t| LossAB: \u001b[91m0.63(+10.75%) ▲\u001b[0m\t\n",
      "Epoch    34 | LossA: \u001b[32m0.56(-1.39%) ▼\u001b[0m\t| LossAB: \u001b[91m1.16(+83.89%) ▲\u001b[0m\t\n",
      "Epoch    35 | LossA: \u001b[91m0.57(+2.40%) ▲\u001b[0m\t| LossAB: \u001b[32m0.39(-66.21%) ▼\u001b[0m\t\n",
      "Epoch    36 | LossA: \u001b[91m0.59(+3.01%) ▲\u001b[0m\t| LossAB: \u001b[91m0.51(+29.52%) ▲\u001b[0m\t\n",
      "Epoch    37 | LossA: \u001b[32m0.55(-7.14%) ▼\u001b[0m\t| LossAB: \u001b[91m0.51(+1.09%) ▲\u001b[0m\t\n",
      "Epoch    38 | LossA: \u001b[32m0.50(-8.72%) ▼\u001b[0m\t| LossAB: \u001b[32m0.46(-10.17%) ▼\u001b[0m\t\n",
      "Epoch    39 | LossA: \u001b[91m0.55(+9.69%) ▲\u001b[0m\t| LossAB: \u001b[91m0.71(+54.47%) ▲\u001b[0m\t\n",
      "Epoch    40 | LossA: \u001b[32m0.54(-0.88%) ▼\u001b[0m\t| LossAB: \u001b[91m0.78(+8.76%) ▲\u001b[0m\t\n",
      "Epoch    41 | LossA: \u001b[91m0.58(+7.11%) ▲\u001b[0m\t| LossAB: \u001b[32m0.62(-19.89%) ▼\u001b[0m\t\n",
      "Epoch    42 | LossA: \u001b[32m0.57(-2.44%) ▼\u001b[0m\t| LossAB: \u001b[91m0.75(+20.10%) ▲\u001b[0m\t\n",
      "Epoch    43 | LossA: \u001b[91m0.58(+2.09%) ▲\u001b[0m\t| LossAB: \u001b[32m0.55(-26.02%) ▼\u001b[0m\t\n",
      "Epoch    44 | LossA: \u001b[32m0.51(-11.66%) ▼\u001b[0m\t| LossAB: \u001b[91m0.65(+17.37%) ▲\u001b[0m\t\n",
      "Epoch    45 | LossA: \u001b[91m0.54(+6.27%) ▲\u001b[0m\t| LossAB: \u001b[32m0.54(-16.94%) ▼\u001b[0m\t\n",
      "Epoch    46 | LossA: \u001b[32m0.48(-11.19%) ▼\u001b[0m\t| LossAB: \u001b[91m0.61(+13.45%) ▲\u001b[0m\t\n",
      "Epoch    47 | LossA: \u001b[91m0.50(+2.92%) ▲\u001b[0m\t| LossAB: \u001b[32m0.45(-26.15%) ▼\u001b[0m\t\n",
      "Epoch    48 | LossA: \u001b[91m0.54(+8.72%) ▲\u001b[0m\t| LossAB: \u001b[91m0.52(+15.04%) ▲\u001b[0m\t\n",
      "654/654 [==============================] - 5s 8ms/step\n",
      "Accuracy: 0.6727828979492188\n"
     ]
    }
   ],
   "source": [
    "#y_test_oh = dense_to_one_hot(y_test, num_clases=4)\n",
    "\n",
    "# visualizing losses and accuracy\n",
    "train_loss = history.history['loss']\n",
    "val_loss = history.history['val_loss']\n",
    "\n",
    "#Observing the losses but can be commented out as it's not mandatory \n",
    "reporter = lossprettifier.LossPrettifier(show_percentage=True)\n",
    "\n",
    "for i in range(numEpochs-1):\n",
    "    reporter(epoch=i, LossA = train_loss[i], LossAB = val_loss[i])\n",
    "\n",
    "# Model evaluation \n",
    "score, acc = model.evaluate(x_test, y_test_oh, batch_size=batch_Size)\n",
    "print(\"Accuracy:\", acc)\n",
    "\n",
    "#if acc>0.675:\n",
    "model.save_weights(modelPath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      0.35      0.51       234\n",
      "           1       0.62      0.96      0.75       246\n",
      "           2       0.74      0.73      0.73       149\n",
      "           3       0.36      0.56      0.44        25\n",
      "\n",
      "    accuracy                           0.67       654\n",
      "   macro avg       0.66      0.65      0.61       654\n",
      "weighted avg       0.74      0.67      0.65       654\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_pred = model.predict(x_test)\n",
    "y_pred = y_pred.reshape(len(y_test), 4)\n",
    "y_pred = np.argmax(y_pred, axis=1)\n",
    "\n",
    "# Writing results on file\n",
    "f = open(resultPath,'a') #create classification report\n",
    "f.write(classification_report(y_test, y_pred))\n",
    "f.write(str(sklm.cohen_kappa_score(y_test, y_pred))+\",\"+str(acc)+\",\"+str(score)+\"\\n\")\n",
    "\n",
    "#Print class-wise classification metrics\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
